{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cartpole DQN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deep Q-Learning Network with Keras and OpenAI Gym, based on [Keon Kim's code](https://github.com/keon/deep-q-learning/blob/master/dqn.py)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Select processing devices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import os\n",
    "# os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import gym\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import Adam\n",
    "import os # for creating directories"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0') # initialise environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_size = env.observation_space.shape[0]\n",
    "state_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "action_size = env.action_space.n\n",
    "action_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_episodes = 1001 # n games we want agent to play (default 1001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output_dir = 'model_output/cartpole/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class DQNAgent:\n",
    "    def __init__(self, state_size, action_size):\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.memory = deque(maxlen=2000) # double-ended queue; acts like list, but elements can be added/removed from either end\n",
    "        self.gamma = 0.95 # decay or discount rate: enables agent to take into account future actions in addition to the immediate ones, but discounted at this rate\n",
    "        self.epsilon = 1.0 # exploration rate: how much to act randomly; more initially than later due to epsilon decay\n",
    "        self.epsilon_decay = 0.995 # decrease number of random explorations as the agent's performance (hopefully) improves over time\n",
    "        self.epsilon_min = 0.01 # minimum amount of random exploration permitted\n",
    "        self.learning_rate = 0.001 # rate at which NN adjusts models parameters via SGD to reduce cost \n",
    "        self.model = self._build_model() # private method \n",
    "    \n",
    "    def _build_model(self):\n",
    "        # neural net to approximate Q-value function:\n",
    "        model = Sequential()\n",
    "        model.add(Dense(24, input_dim=self.state_size, activation='relu')) # 1st hidden layer; states as input\n",
    "        model.add(Dense(24, activation='relu')) # 2nd hidden layer\n",
    "        model.add(Dense(self.action_size, activation='linear')) # 2 actions, so 2 output neurons: 0 and 1 (L/R)\n",
    "        model.compile(loss='mse',\n",
    "                      optimizer=Adam(lr=self.learning_rate))\n",
    "        return model\n",
    "    \n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        self.memory.append((state, action, reward, next_state, done)) # list of previous experiences, enabling re-training later\n",
    "\n",
    "    def act(self, state):\n",
    "        if np.random.rand() <= self.epsilon: # if acting randomly, take random action\n",
    "            return random.randrange(self.action_size)\n",
    "        act_values = self.model.predict(state) # if not acting randomly, predict reward value based on current state\n",
    "        return np.argmax(act_values[0]) # pick the action that will give the highest reward (i.e., go left or right?)\n",
    "\n",
    "    def replay(self, batch_size): # method that trains NN with experiences sampled from memory\n",
    "        minibatch = random.sample(self.memory, batch_size) # sample a minibatch from memory\n",
    "        for state, action, reward, next_state, done in minibatch: # extract data for each minibatch sample\n",
    "            target = reward # if done (boolean whether game ended or not, i.e., whether final state or not), then target = reward\n",
    "            if not done: # if not done, then predict future discounted reward\n",
    "                target = (reward + self.gamma * # (target) = reward + (discount rate gamma) * \n",
    "                          np.amax(self.model.predict(next_state)[0])) # (maximum target Q based on future action a')\n",
    "            target_f = self.model.predict(state) # approximately map current state to future discounted reward\n",
    "            target_f[0][action] = target\n",
    "            self.model.fit(state, target_f, epochs=1, verbose=0) # single epoch of training with x=state, y=target_f; fit decreases loss btwn target_f and y_hat\n",
    "        if self.epsilon > self.epsilon_min:\n",
    "            self.epsilon *= self.epsilon_decay\n",
    "\n",
    "    def load(self, name):\n",
    "        self.model.load_weights(name)\n",
    "\n",
    "    def save(self, name):\n",
    "        self.model.save_weights(name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Interact with environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = DQNAgent(state_size, action_size) # initialise agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 0/1001, score: 8, e: 1.0\n",
      "episode: 1/1001, score: 19, e: 1.0\n",
      "episode: 2/1001, score: 27, e: 1.0\n",
      "episode: 3/1001, score: 15, e: 0.99\n",
      "episode: 4/1001, score: 16, e: 0.99\n",
      "episode: 5/1001, score: 12, e: 0.99\n",
      "episode: 6/1001, score: 14, e: 0.98\n",
      "episode: 7/1001, score: 14, e: 0.98\n",
      "episode: 8/1001, score: 34, e: 0.97\n",
      "episode: 9/1001, score: 21, e: 0.97\n",
      "episode: 10/1001, score: 21, e: 0.96\n",
      "episode: 11/1001, score: 20, e: 0.96\n",
      "episode: 12/1001, score: 29, e: 0.95\n",
      "episode: 13/1001, score: 15, e: 0.95\n",
      "episode: 14/1001, score: 14, e: 0.94\n",
      "episode: 15/1001, score: 40, e: 0.94\n",
      "episode: 16/1001, score: 12, e: 0.93\n",
      "episode: 17/1001, score: 16, e: 0.93\n",
      "episode: 18/1001, score: 19, e: 0.92\n",
      "episode: 19/1001, score: 27, e: 0.92\n",
      "episode: 20/1001, score: 20, e: 0.91\n",
      "episode: 21/1001, score: 16, e: 0.91\n",
      "episode: 22/1001, score: 13, e: 0.9\n",
      "episode: 23/1001, score: 22, e: 0.9\n",
      "episode: 24/1001, score: 19, e: 0.9\n",
      "episode: 25/1001, score: 19, e: 0.89\n",
      "episode: 26/1001, score: 34, e: 0.89\n",
      "episode: 27/1001, score: 13, e: 0.88\n",
      "episode: 28/1001, score: 26, e: 0.88\n",
      "episode: 29/1001, score: 17, e: 0.87\n",
      "episode: 30/1001, score: 9, e: 0.87\n",
      "episode: 31/1001, score: 41, e: 0.86\n",
      "episode: 32/1001, score: 60, e: 0.86\n",
      "episode: 33/1001, score: 10, e: 0.86\n",
      "episode: 34/1001, score: 12, e: 0.85\n",
      "episode: 35/1001, score: 14, e: 0.85\n",
      "episode: 36/1001, score: 23, e: 0.84\n",
      "episode: 37/1001, score: 31, e: 0.84\n",
      "episode: 38/1001, score: 32, e: 0.83\n",
      "episode: 39/1001, score: 13, e: 0.83\n",
      "episode: 40/1001, score: 23, e: 0.83\n",
      "episode: 41/1001, score: 24, e: 0.82\n",
      "episode: 42/1001, score: 26, e: 0.82\n",
      "episode: 43/1001, score: 11, e: 0.81\n",
      "episode: 44/1001, score: 17, e: 0.81\n",
      "episode: 45/1001, score: 14, e: 0.81\n",
      "episode: 46/1001, score: 23, e: 0.8\n",
      "episode: 47/1001, score: 9, e: 0.8\n",
      "episode: 48/1001, score: 25, e: 0.79\n",
      "episode: 49/1001, score: 16, e: 0.79\n",
      "episode: 50/1001, score: 20, e: 0.79\n",
      "episode: 51/1001, score: 14, e: 0.78\n",
      "episode: 52/1001, score: 15, e: 0.78\n",
      "episode: 53/1001, score: 37, e: 0.77\n",
      "episode: 54/1001, score: 14, e: 0.77\n",
      "episode: 55/1001, score: 28, e: 0.77\n",
      "episode: 56/1001, score: 66, e: 0.76\n",
      "episode: 57/1001, score: 11, e: 0.76\n",
      "episode: 58/1001, score: 48, e: 0.76\n",
      "episode: 59/1001, score: 168, e: 0.75\n",
      "episode: 60/1001, score: 17, e: 0.75\n",
      "episode: 61/1001, score: 67, e: 0.74\n",
      "episode: 62/1001, score: 34, e: 0.74\n",
      "episode: 63/1001, score: 8, e: 0.74\n",
      "episode: 64/1001, score: 17, e: 0.73\n",
      "episode: 65/1001, score: 29, e: 0.73\n",
      "episode: 66/1001, score: 14, e: 0.73\n",
      "episode: 67/1001, score: 16, e: 0.72\n",
      "episode: 68/1001, score: 35, e: 0.72\n",
      "episode: 69/1001, score: 42, e: 0.71\n",
      "episode: 70/1001, score: 34, e: 0.71\n",
      "episode: 71/1001, score: 13, e: 0.71\n",
      "episode: 72/1001, score: 13, e: 0.7\n",
      "episode: 73/1001, score: 10, e: 0.7\n",
      "episode: 74/1001, score: 27, e: 0.7\n",
      "episode: 75/1001, score: 15, e: 0.69\n",
      "episode: 76/1001, score: 15, e: 0.69\n",
      "episode: 77/1001, score: 17, e: 0.69\n",
      "episode: 78/1001, score: 7, e: 0.68\n",
      "episode: 79/1001, score: 8, e: 0.68\n",
      "episode: 80/1001, score: 11, e: 0.68\n",
      "episode: 81/1001, score: 19, e: 0.67\n",
      "episode: 82/1001, score: 14, e: 0.67\n",
      "episode: 83/1001, score: 15, e: 0.67\n",
      "episode: 84/1001, score: 16, e: 0.66\n",
      "episode: 85/1001, score: 18, e: 0.66\n",
      "episode: 86/1001, score: 18, e: 0.66\n",
      "episode: 87/1001, score: 24, e: 0.65\n",
      "episode: 88/1001, score: 14, e: 0.65\n",
      "episode: 89/1001, score: 27, e: 0.65\n",
      "episode: 90/1001, score: 26, e: 0.64\n",
      "episode: 91/1001, score: 17, e: 0.64\n",
      "episode: 92/1001, score: 28, e: 0.64\n",
      "episode: 93/1001, score: 15, e: 0.63\n",
      "episode: 94/1001, score: 10, e: 0.63\n",
      "episode: 95/1001, score: 16, e: 0.63\n",
      "episode: 96/1001, score: 9, e: 0.62\n",
      "episode: 97/1001, score: 9, e: 0.62\n",
      "episode: 98/1001, score: 12, e: 0.62\n",
      "episode: 99/1001, score: 15, e: 0.61\n",
      "episode: 100/1001, score: 17, e: 0.61\n",
      "episode: 101/1001, score: 42, e: 0.61\n",
      "episode: 102/1001, score: 15, e: 0.61\n",
      "episode: 103/1001, score: 51, e: 0.6\n",
      "episode: 104/1001, score: 16, e: 0.6\n",
      "episode: 105/1001, score: 22, e: 0.6\n",
      "episode: 106/1001, score: 13, e: 0.59\n",
      "episode: 107/1001, score: 19, e: 0.59\n",
      "episode: 108/1001, score: 17, e: 0.59\n",
      "episode: 109/1001, score: 10, e: 0.58\n",
      "episode: 110/1001, score: 15, e: 0.58\n",
      "episode: 111/1001, score: 8, e: 0.58\n",
      "episode: 112/1001, score: 13, e: 0.58\n",
      "episode: 113/1001, score: 9, e: 0.57\n",
      "episode: 114/1001, score: 19, e: 0.57\n",
      "episode: 115/1001, score: 10, e: 0.57\n",
      "episode: 116/1001, score: 24, e: 0.56\n",
      "episode: 117/1001, score: 13, e: 0.56\n",
      "episode: 118/1001, score: 8, e: 0.56\n",
      "episode: 119/1001, score: 17, e: 0.56\n",
      "episode: 120/1001, score: 10, e: 0.55\n",
      "episode: 121/1001, score: 13, e: 0.55\n",
      "episode: 122/1001, score: 12, e: 0.55\n",
      "episode: 123/1001, score: 13, e: 0.55\n",
      "episode: 124/1001, score: 12, e: 0.54\n",
      "episode: 125/1001, score: 9, e: 0.54\n",
      "episode: 126/1001, score: 15, e: 0.54\n",
      "episode: 127/1001, score: 8, e: 0.53\n",
      "episode: 128/1001, score: 10, e: 0.53\n",
      "episode: 129/1001, score: 13, e: 0.53\n",
      "episode: 130/1001, score: 10, e: 0.53\n",
      "episode: 131/1001, score: 9, e: 0.52\n",
      "episode: 132/1001, score: 9, e: 0.52\n",
      "episode: 133/1001, score: 9, e: 0.52\n",
      "episode: 134/1001, score: 11, e: 0.52\n",
      "episode: 135/1001, score: 17, e: 0.51\n",
      "episode: 136/1001, score: 16, e: 0.51\n",
      "episode: 137/1001, score: 12, e: 0.51\n",
      "episode: 138/1001, score: 11, e: 0.51\n",
      "episode: 139/1001, score: 13, e: 0.5\n",
      "episode: 140/1001, score: 12, e: 0.5\n",
      "episode: 141/1001, score: 12, e: 0.5\n",
      "episode: 142/1001, score: 49, e: 0.5\n",
      "episode: 143/1001, score: 62, e: 0.49\n",
      "episode: 144/1001, score: 57, e: 0.49\n",
      "episode: 145/1001, score: 26, e: 0.49\n",
      "episode: 146/1001, score: 51, e: 0.49\n",
      "episode: 147/1001, score: 27, e: 0.48\n",
      "episode: 148/1001, score: 22, e: 0.48\n",
      "episode: 149/1001, score: 44, e: 0.48\n",
      "episode: 150/1001, score: 29, e: 0.48\n",
      "episode: 151/1001, score: 73, e: 0.47\n",
      "episode: 152/1001, score: 37, e: 0.47\n",
      "episode: 153/1001, score: 21, e: 0.47\n",
      "episode: 154/1001, score: 38, e: 0.47\n",
      "episode: 155/1001, score: 35, e: 0.46\n",
      "episode: 156/1001, score: 31, e: 0.46\n",
      "episode: 157/1001, score: 54, e: 0.46\n",
      "episode: 158/1001, score: 25, e: 0.46\n",
      "episode: 159/1001, score: 44, e: 0.46\n",
      "episode: 160/1001, score: 44, e: 0.45\n",
      "episode: 161/1001, score: 95, e: 0.45\n",
      "episode: 162/1001, score: 24, e: 0.45\n",
      "episode: 163/1001, score: 16, e: 0.45\n",
      "episode: 164/1001, score: 71, e: 0.44\n",
      "episode: 165/1001, score: 41, e: 0.44\n",
      "episode: 166/1001, score: 18, e: 0.44\n",
      "episode: 167/1001, score: 27, e: 0.44\n",
      "episode: 168/1001, score: 35, e: 0.44\n",
      "episode: 169/1001, score: 67, e: 0.43\n",
      "episode: 170/1001, score: 20, e: 0.43\n",
      "episode: 171/1001, score: 24, e: 0.43\n",
      "episode: 172/1001, score: 24, e: 0.43\n",
      "episode: 173/1001, score: 60, e: 0.42\n",
      "episode: 174/1001, score: 62, e: 0.42\n",
      "episode: 175/1001, score: 32, e: 0.42\n",
      "episode: 176/1001, score: 37, e: 0.42\n",
      "episode: 177/1001, score: 54, e: 0.42\n",
      "episode: 178/1001, score: 53, e: 0.41\n",
      "episode: 179/1001, score: 26, e: 0.41\n",
      "episode: 180/1001, score: 25, e: 0.41\n",
      "episode: 181/1001, score: 26, e: 0.41\n",
      "episode: 182/1001, score: 20, e: 0.41\n",
      "episode: 183/1001, score: 19, e: 0.4\n",
      "episode: 184/1001, score: 18, e: 0.4\n",
      "episode: 185/1001, score: 32, e: 0.4\n",
      "episode: 186/1001, score: 29, e: 0.4\n",
      "episode: 187/1001, score: 74, e: 0.4\n",
      "episode: 188/1001, score: 32, e: 0.39\n",
      "episode: 189/1001, score: 125, e: 0.39\n",
      "episode: 190/1001, score: 85, e: 0.39\n",
      "episode: 191/1001, score: 22, e: 0.39\n",
      "episode: 192/1001, score: 61, e: 0.39\n",
      "episode: 193/1001, score: 23, e: 0.38\n",
      "episode: 194/1001, score: 26, e: 0.38\n",
      "episode: 195/1001, score: 30, e: 0.38\n",
      "episode: 196/1001, score: 65, e: 0.38\n",
      "episode: 197/1001, score: 75, e: 0.38\n",
      "episode: 198/1001, score: 39, e: 0.37\n",
      "episode: 199/1001, score: 51, e: 0.37\n",
      "episode: 200/1001, score: 41, e: 0.37\n",
      "episode: 201/1001, score: 27, e: 0.37\n",
      "episode: 202/1001, score: 30, e: 0.37\n",
      "episode: 203/1001, score: 35, e: 0.37\n",
      "episode: 204/1001, score: 40, e: 0.36\n",
      "episode: 205/1001, score: 49, e: 0.36\n",
      "episode: 206/1001, score: 57, e: 0.36\n",
      "episode: 207/1001, score: 17, e: 0.36\n",
      "episode: 208/1001, score: 61, e: 0.36\n",
      "episode: 209/1001, score: 79, e: 0.35\n",
      "episode: 210/1001, score: 44, e: 0.35\n",
      "episode: 211/1001, score: 25, e: 0.35\n",
      "episode: 212/1001, score: 58, e: 0.35\n",
      "episode: 213/1001, score: 38, e: 0.35\n",
      "episode: 214/1001, score: 31, e: 0.35\n",
      "episode: 215/1001, score: 94, e: 0.34\n",
      "episode: 216/1001, score: 74, e: 0.34\n",
      "episode: 217/1001, score: 147, e: 0.34\n",
      "episode: 218/1001, score: 48, e: 0.34\n",
      "episode: 219/1001, score: 122, e: 0.34\n",
      "episode: 220/1001, score: 48, e: 0.34\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 221/1001, score: 31, e: 0.33\n",
      "episode: 222/1001, score: 68, e: 0.33\n",
      "episode: 223/1001, score: 102, e: 0.33\n",
      "episode: 224/1001, score: 101, e: 0.33\n",
      "episode: 225/1001, score: 55, e: 0.33\n",
      "episode: 226/1001, score: 75, e: 0.33\n",
      "episode: 227/1001, score: 84, e: 0.32\n",
      "episode: 228/1001, score: 43, e: 0.32\n",
      "episode: 229/1001, score: 48, e: 0.32\n",
      "episode: 230/1001, score: 90, e: 0.32\n",
      "episode: 231/1001, score: 75, e: 0.32\n",
      "episode: 232/1001, score: 40, e: 0.32\n",
      "episode: 233/1001, score: 58, e: 0.31\n",
      "episode: 234/1001, score: 55, e: 0.31\n",
      "episode: 235/1001, score: 51, e: 0.31\n",
      "episode: 236/1001, score: 103, e: 0.31\n",
      "episode: 237/1001, score: 76, e: 0.31\n",
      "episode: 238/1001, score: 43, e: 0.31\n",
      "episode: 239/1001, score: 21, e: 0.3\n",
      "episode: 240/1001, score: 40, e: 0.3\n",
      "episode: 241/1001, score: 68, e: 0.3\n",
      "episode: 242/1001, score: 55, e: 0.3\n",
      "episode: 243/1001, score: 66, e: 0.3\n",
      "episode: 244/1001, score: 54, e: 0.3\n",
      "episode: 245/1001, score: 68, e: 0.3\n",
      "episode: 246/1001, score: 119, e: 0.29\n",
      "episode: 247/1001, score: 30, e: 0.29\n",
      "episode: 248/1001, score: 58, e: 0.29\n",
      "episode: 249/1001, score: 67, e: 0.29\n",
      "episode: 250/1001, score: 66, e: 0.29\n",
      "episode: 251/1001, score: 52, e: 0.29\n",
      "episode: 252/1001, score: 199, e: 0.29\n",
      "episode: 253/1001, score: 91, e: 0.28\n",
      "episode: 254/1001, score: 43, e: 0.28\n",
      "episode: 255/1001, score: 64, e: 0.28\n",
      "episode: 256/1001, score: 66, e: 0.28\n",
      "episode: 257/1001, score: 42, e: 0.28\n",
      "episode: 258/1001, score: 31, e: 0.28\n",
      "episode: 259/1001, score: 22, e: 0.28\n",
      "episode: 260/1001, score: 74, e: 0.27\n",
      "episode: 261/1001, score: 41, e: 0.27\n",
      "episode: 262/1001, score: 60, e: 0.27\n",
      "episode: 263/1001, score: 78, e: 0.27\n",
      "episode: 264/1001, score: 39, e: 0.27\n",
      "episode: 265/1001, score: 30, e: 0.27\n",
      "episode: 266/1001, score: 63, e: 0.27\n",
      "episode: 267/1001, score: 46, e: 0.26\n",
      "episode: 268/1001, score: 64, e: 0.26\n",
      "episode: 269/1001, score: 53, e: 0.26\n",
      "episode: 270/1001, score: 45, e: 0.26\n",
      "episode: 271/1001, score: 30, e: 0.26\n",
      "episode: 272/1001, score: 112, e: 0.26\n",
      "episode: 273/1001, score: 110, e: 0.26\n",
      "episode: 274/1001, score: 69, e: 0.26\n",
      "episode: 275/1001, score: 155, e: 0.25\n",
      "episode: 276/1001, score: 77, e: 0.25\n",
      "episode: 277/1001, score: 109, e: 0.25\n",
      "episode: 278/1001, score: 55, e: 0.25\n",
      "episode: 279/1001, score: 79, e: 0.25\n",
      "episode: 280/1001, score: 137, e: 0.25\n",
      "episode: 281/1001, score: 71, e: 0.25\n",
      "episode: 282/1001, score: 106, e: 0.25\n",
      "episode: 283/1001, score: 111, e: 0.24\n",
      "episode: 284/1001, score: 49, e: 0.24\n",
      "episode: 285/1001, score: 65, e: 0.24\n",
      "episode: 286/1001, score: 36, e: 0.24\n",
      "episode: 287/1001, score: 74, e: 0.24\n",
      "episode: 288/1001, score: 63, e: 0.24\n",
      "episode: 289/1001, score: 67, e: 0.24\n",
      "episode: 290/1001, score: 138, e: 0.24\n",
      "episode: 291/1001, score: 58, e: 0.23\n",
      "episode: 292/1001, score: 70, e: 0.23\n",
      "episode: 293/1001, score: 99, e: 0.23\n",
      "episode: 294/1001, score: 125, e: 0.23\n",
      "episode: 295/1001, score: 77, e: 0.23\n",
      "episode: 296/1001, score: 147, e: 0.23\n",
      "episode: 297/1001, score: 74, e: 0.23\n",
      "episode: 298/1001, score: 128, e: 0.23\n",
      "episode: 299/1001, score: 102, e: 0.23\n",
      "episode: 300/1001, score: 74, e: 0.22\n",
      "episode: 301/1001, score: 87, e: 0.22\n",
      "episode: 302/1001, score: 188, e: 0.22\n",
      "episode: 303/1001, score: 92, e: 0.22\n",
      "episode: 304/1001, score: 71, e: 0.22\n",
      "episode: 305/1001, score: 34, e: 0.22\n",
      "episode: 306/1001, score: 75, e: 0.22\n",
      "episode: 307/1001, score: 85, e: 0.22\n",
      "episode: 308/1001, score: 89, e: 0.22\n",
      "episode: 309/1001, score: 44, e: 0.21\n",
      "episode: 310/1001, score: 16, e: 0.21\n",
      "episode: 311/1001, score: 97, e: 0.21\n",
      "episode: 312/1001, score: 51, e: 0.21\n",
      "episode: 313/1001, score: 73, e: 0.21\n",
      "episode: 314/1001, score: 53, e: 0.21\n",
      "episode: 315/1001, score: 199, e: 0.21\n",
      "episode: 316/1001, score: 84, e: 0.21\n",
      "episode: 317/1001, score: 123, e: 0.21\n",
      "episode: 318/1001, score: 90, e: 0.21\n",
      "episode: 319/1001, score: 103, e: 0.2\n",
      "episode: 320/1001, score: 58, e: 0.2\n",
      "episode: 321/1001, score: 37, e: 0.2\n",
      "episode: 322/1001, score: 54, e: 0.2\n",
      "episode: 323/1001, score: 78, e: 0.2\n",
      "episode: 324/1001, score: 199, e: 0.2\n",
      "episode: 325/1001, score: 135, e: 0.2\n",
      "episode: 326/1001, score: 18, e: 0.2\n",
      "episode: 327/1001, score: 34, e: 0.2\n",
      "episode: 328/1001, score: 32, e: 0.2\n",
      "episode: 329/1001, score: 32, e: 0.19\n",
      "episode: 330/1001, score: 131, e: 0.19\n",
      "episode: 331/1001, score: 91, e: 0.19\n",
      "episode: 332/1001, score: 81, e: 0.19\n",
      "episode: 333/1001, score: 183, e: 0.19\n",
      "episode: 334/1001, score: 105, e: 0.19\n",
      "episode: 335/1001, score: 73, e: 0.19\n",
      "episode: 336/1001, score: 63, e: 0.19\n",
      "episode: 337/1001, score: 67, e: 0.19\n",
      "episode: 338/1001, score: 55, e: 0.19\n",
      "episode: 339/1001, score: 71, e: 0.18\n",
      "episode: 340/1001, score: 71, e: 0.18\n",
      "episode: 341/1001, score: 85, e: 0.18\n",
      "episode: 342/1001, score: 56, e: 0.18\n",
      "episode: 343/1001, score: 54, e: 0.18\n",
      "episode: 344/1001, score: 60, e: 0.18\n",
      "episode: 345/1001, score: 49, e: 0.18\n",
      "episode: 346/1001, score: 88, e: 0.18\n",
      "episode: 347/1001, score: 83, e: 0.18\n",
      "episode: 348/1001, score: 47, e: 0.18\n",
      "episode: 349/1001, score: 104, e: 0.18\n",
      "episode: 350/1001, score: 166, e: 0.17\n",
      "episode: 351/1001, score: 97, e: 0.17\n",
      "episode: 352/1001, score: 152, e: 0.17\n",
      "episode: 353/1001, score: 43, e: 0.17\n",
      "episode: 354/1001, score: 191, e: 0.17\n",
      "episode: 355/1001, score: 199, e: 0.17\n",
      "episode: 356/1001, score: 197, e: 0.17\n",
      "episode: 357/1001, score: 140, e: 0.17\n",
      "episode: 358/1001, score: 133, e: 0.17\n",
      "episode: 359/1001, score: 151, e: 0.17\n",
      "episode: 360/1001, score: 177, e: 0.17\n",
      "episode: 361/1001, score: 199, e: 0.17\n",
      "episode: 362/1001, score: 151, e: 0.16\n",
      "episode: 363/1001, score: 120, e: 0.16\n",
      "episode: 364/1001, score: 139, e: 0.16\n",
      "episode: 365/1001, score: 113, e: 0.16\n",
      "episode: 366/1001, score: 199, e: 0.16\n",
      "episode: 367/1001, score: 80, e: 0.16\n",
      "episode: 368/1001, score: 158, e: 0.16\n",
      "episode: 369/1001, score: 46, e: 0.16\n",
      "episode: 370/1001, score: 34, e: 0.16\n",
      "episode: 371/1001, score: 44, e: 0.16\n",
      "episode: 372/1001, score: 175, e: 0.16\n",
      "episode: 373/1001, score: 199, e: 0.16\n",
      "episode: 374/1001, score: 79, e: 0.15\n",
      "episode: 375/1001, score: 62, e: 0.15\n",
      "episode: 376/1001, score: 67, e: 0.15\n",
      "episode: 377/1001, score: 108, e: 0.15\n",
      "episode: 378/1001, score: 79, e: 0.15\n",
      "episode: 379/1001, score: 199, e: 0.15\n",
      "episode: 380/1001, score: 97, e: 0.15\n",
      "episode: 381/1001, score: 50, e: 0.15\n",
      "episode: 382/1001, score: 33, e: 0.15\n",
      "episode: 383/1001, score: 48, e: 0.15\n",
      "episode: 384/1001, score: 199, e: 0.15\n",
      "episode: 385/1001, score: 199, e: 0.15\n",
      "episode: 386/1001, score: 126, e: 0.15\n",
      "episode: 387/1001, score: 97, e: 0.15\n",
      "episode: 388/1001, score: 67, e: 0.14\n",
      "episode: 389/1001, score: 35, e: 0.14\n",
      "episode: 390/1001, score: 60, e: 0.14\n",
      "episode: 391/1001, score: 168, e: 0.14\n",
      "episode: 392/1001, score: 150, e: 0.14\n",
      "episode: 393/1001, score: 137, e: 0.14\n",
      "episode: 394/1001, score: 199, e: 0.14\n",
      "episode: 395/1001, score: 102, e: 0.14\n",
      "episode: 396/1001, score: 138, e: 0.14\n",
      "episode: 397/1001, score: 128, e: 0.14\n",
      "episode: 398/1001, score: 159, e: 0.14\n",
      "episode: 399/1001, score: 157, e: 0.14\n",
      "episode: 400/1001, score: 110, e: 0.14\n",
      "episode: 401/1001, score: 137, e: 0.14\n",
      "episode: 402/1001, score: 189, e: 0.13\n",
      "episode: 403/1001, score: 147, e: 0.13\n",
      "episode: 404/1001, score: 199, e: 0.13\n",
      "episode: 405/1001, score: 69, e: 0.13\n",
      "episode: 406/1001, score: 73, e: 0.13\n",
      "episode: 407/1001, score: 69, e: 0.13\n",
      "episode: 408/1001, score: 112, e: 0.13\n",
      "episode: 409/1001, score: 127, e: 0.13\n",
      "episode: 410/1001, score: 132, e: 0.13\n",
      "episode: 411/1001, score: 199, e: 0.13\n",
      "episode: 412/1001, score: 129, e: 0.13\n",
      "episode: 413/1001, score: 160, e: 0.13\n",
      "episode: 414/1001, score: 98, e: 0.13\n",
      "episode: 415/1001, score: 199, e: 0.13\n",
      "episode: 416/1001, score: 72, e: 0.13\n",
      "episode: 417/1001, score: 103, e: 0.12\n",
      "episode: 418/1001, score: 133, e: 0.12\n",
      "episode: 419/1001, score: 36, e: 0.12\n",
      "episode: 420/1001, score: 30, e: 0.12\n",
      "episode: 421/1001, score: 49, e: 0.12\n",
      "episode: 422/1001, score: 52, e: 0.12\n",
      "episode: 423/1001, score: 101, e: 0.12\n",
      "episode: 424/1001, score: 183, e: 0.12\n",
      "episode: 425/1001, score: 65, e: 0.12\n",
      "episode: 426/1001, score: 68, e: 0.12\n",
      "episode: 427/1001, score: 74, e: 0.12\n",
      "episode: 428/1001, score: 199, e: 0.12\n",
      "episode: 429/1001, score: 57, e: 0.12\n",
      "episode: 430/1001, score: 93, e: 0.12\n",
      "episode: 431/1001, score: 86, e: 0.12\n",
      "episode: 432/1001, score: 83, e: 0.12\n",
      "episode: 433/1001, score: 130, e: 0.12\n",
      "episode: 434/1001, score: 199, e: 0.11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 435/1001, score: 110, e: 0.11\n",
      "episode: 436/1001, score: 137, e: 0.11\n",
      "episode: 437/1001, score: 89, e: 0.11\n",
      "episode: 438/1001, score: 154, e: 0.11\n",
      "episode: 439/1001, score: 126, e: 0.11\n",
      "episode: 440/1001, score: 56, e: 0.11\n",
      "episode: 441/1001, score: 44, e: 0.11\n",
      "episode: 442/1001, score: 182, e: 0.11\n",
      "episode: 443/1001, score: 45, e: 0.11\n",
      "episode: 444/1001, score: 63, e: 0.11\n",
      "episode: 445/1001, score: 36, e: 0.11\n",
      "episode: 446/1001, score: 44, e: 0.11\n",
      "episode: 447/1001, score: 92, e: 0.11\n",
      "episode: 448/1001, score: 150, e: 0.11\n",
      "episode: 449/1001, score: 38, e: 0.11\n",
      "episode: 450/1001, score: 61, e: 0.11\n",
      "episode: 451/1001, score: 106, e: 0.11\n",
      "episode: 452/1001, score: 155, e: 0.1\n",
      "episode: 453/1001, score: 199, e: 0.1\n",
      "episode: 454/1001, score: 45, e: 0.1\n",
      "episode: 455/1001, score: 112, e: 0.1\n",
      "episode: 456/1001, score: 96, e: 0.1\n",
      "episode: 457/1001, score: 156, e: 0.1\n",
      "episode: 458/1001, score: 85, e: 0.1\n",
      "episode: 459/1001, score: 183, e: 0.1\n",
      "episode: 460/1001, score: 199, e: 0.1\n",
      "episode: 461/1001, score: 168, e: 0.1\n",
      "episode: 462/1001, score: 34, e: 0.1\n",
      "episode: 463/1001, score: 199, e: 0.099\n",
      "episode: 464/1001, score: 62, e: 0.099\n",
      "episode: 465/1001, score: 63, e: 0.098\n",
      "episode: 466/1001, score: 69, e: 0.098\n",
      "episode: 467/1001, score: 68, e: 0.097\n",
      "episode: 468/1001, score: 48, e: 0.097\n",
      "episode: 469/1001, score: 43, e: 0.096\n",
      "episode: 470/1001, score: 63, e: 0.096\n",
      "episode: 471/1001, score: 76, e: 0.095\n",
      "episode: 472/1001, score: 77, e: 0.095\n",
      "episode: 473/1001, score: 58, e: 0.094\n",
      "episode: 474/1001, score: 55, e: 0.094\n",
      "episode: 475/1001, score: 39, e: 0.093\n",
      "episode: 476/1001, score: 67, e: 0.093\n",
      "episode: 477/1001, score: 50, e: 0.092\n",
      "episode: 478/1001, score: 51, e: 0.092\n",
      "episode: 479/1001, score: 56, e: 0.092\n",
      "episode: 480/1001, score: 82, e: 0.091\n",
      "episode: 481/1001, score: 63, e: 0.091\n",
      "episode: 482/1001, score: 64, e: 0.09\n",
      "episode: 483/1001, score: 67, e: 0.09\n",
      "episode: 484/1001, score: 37, e: 0.089\n",
      "episode: 485/1001, score: 59, e: 0.089\n",
      "episode: 486/1001, score: 61, e: 0.088\n",
      "episode: 487/1001, score: 85, e: 0.088\n",
      "episode: 488/1001, score: 62, e: 0.088\n",
      "episode: 489/1001, score: 31, e: 0.087\n",
      "episode: 490/1001, score: 48, e: 0.087\n",
      "episode: 491/1001, score: 74, e: 0.086\n",
      "episode: 492/1001, score: 70, e: 0.086\n",
      "episode: 493/1001, score: 88, e: 0.085\n",
      "episode: 494/1001, score: 86, e: 0.085\n",
      "episode: 495/1001, score: 199, e: 0.084\n",
      "episode: 496/1001, score: 92, e: 0.084\n",
      "episode: 497/1001, score: 109, e: 0.084\n",
      "episode: 498/1001, score: 84, e: 0.083\n",
      "episode: 499/1001, score: 93, e: 0.083\n",
      "episode: 500/1001, score: 84, e: 0.082\n",
      "episode: 501/1001, score: 86, e: 0.082\n",
      "episode: 502/1001, score: 76, e: 0.082\n",
      "episode: 503/1001, score: 55, e: 0.081\n",
      "episode: 504/1001, score: 96, e: 0.081\n",
      "episode: 505/1001, score: 132, e: 0.08\n",
      "episode: 506/1001, score: 61, e: 0.08\n",
      "episode: 507/1001, score: 100, e: 0.08\n",
      "episode: 508/1001, score: 93, e: 0.079\n",
      "episode: 509/1001, score: 57, e: 0.079\n",
      "episode: 510/1001, score: 62, e: 0.078\n",
      "episode: 511/1001, score: 89, e: 0.078\n",
      "episode: 512/1001, score: 71, e: 0.078\n",
      "episode: 513/1001, score: 102, e: 0.077\n",
      "episode: 514/1001, score: 199, e: 0.077\n",
      "episode: 515/1001, score: 82, e: 0.076\n",
      "episode: 516/1001, score: 95, e: 0.076\n",
      "episode: 517/1001, score: 175, e: 0.076\n",
      "episode: 518/1001, score: 83, e: 0.075\n",
      "episode: 519/1001, score: 81, e: 0.075\n",
      "episode: 520/1001, score: 79, e: 0.075\n",
      "episode: 521/1001, score: 70, e: 0.074\n",
      "episode: 522/1001, score: 78, e: 0.074\n",
      "episode: 523/1001, score: 62, e: 0.073\n",
      "episode: 524/1001, score: 92, e: 0.073\n",
      "episode: 525/1001, score: 80, e: 0.073\n",
      "episode: 526/1001, score: 72, e: 0.072\n",
      "episode: 527/1001, score: 77, e: 0.072\n",
      "episode: 528/1001, score: 66, e: 0.072\n",
      "episode: 529/1001, score: 49, e: 0.071\n",
      "episode: 530/1001, score: 74, e: 0.071\n",
      "episode: 531/1001, score: 145, e: 0.071\n",
      "episode: 532/1001, score: 62, e: 0.07\n",
      "episode: 533/1001, score: 163, e: 0.07\n",
      "episode: 534/1001, score: 71, e: 0.069\n",
      "episode: 535/1001, score: 68, e: 0.069\n",
      "episode: 536/1001, score: 95, e: 0.069\n",
      "episode: 537/1001, score: 58, e: 0.068\n",
      "episode: 538/1001, score: 101, e: 0.068\n",
      "episode: 539/1001, score: 55, e: 0.068\n",
      "episode: 540/1001, score: 56, e: 0.067\n",
      "episode: 541/1001, score: 170, e: 0.067\n",
      "episode: 542/1001, score: 40, e: 0.067\n",
      "episode: 543/1001, score: 199, e: 0.066\n",
      "episode: 544/1001, score: 89, e: 0.066\n",
      "episode: 545/1001, score: 78, e: 0.066\n",
      "episode: 546/1001, score: 62, e: 0.065\n",
      "episode: 547/1001, score: 50, e: 0.065\n",
      "episode: 548/1001, score: 47, e: 0.065\n",
      "episode: 549/1001, score: 42, e: 0.064\n",
      "episode: 550/1001, score: 55, e: 0.064\n",
      "episode: 551/1001, score: 75, e: 0.064\n",
      "episode: 552/1001, score: 93, e: 0.063\n",
      "episode: 553/1001, score: 43, e: 0.063\n",
      "episode: 554/1001, score: 45, e: 0.063\n",
      "episode: 555/1001, score: 38, e: 0.063\n",
      "episode: 556/1001, score: 77, e: 0.062\n",
      "episode: 557/1001, score: 82, e: 0.062\n",
      "episode: 558/1001, score: 54, e: 0.062\n",
      "episode: 559/1001, score: 36, e: 0.061\n",
      "episode: 560/1001, score: 27, e: 0.061\n",
      "episode: 561/1001, score: 58, e: 0.061\n",
      "episode: 562/1001, score: 30, e: 0.06\n",
      "episode: 563/1001, score: 38, e: 0.06\n",
      "episode: 564/1001, score: 34, e: 0.06\n",
      "episode: 565/1001, score: 32, e: 0.059\n",
      "episode: 566/1001, score: 35, e: 0.059\n",
      "episode: 567/1001, score: 35, e: 0.059\n",
      "episode: 568/1001, score: 33, e: 0.059\n",
      "episode: 569/1001, score: 41, e: 0.058\n",
      "episode: 570/1001, score: 32, e: 0.058\n",
      "episode: 571/1001, score: 32, e: 0.058\n",
      "episode: 572/1001, score: 31, e: 0.057\n",
      "episode: 573/1001, score: 27, e: 0.057\n",
      "episode: 574/1001, score: 30, e: 0.057\n",
      "episode: 575/1001, score: 56, e: 0.057\n",
      "episode: 576/1001, score: 48, e: 0.056\n",
      "episode: 577/1001, score: 57, e: 0.056\n",
      "episode: 578/1001, score: 57, e: 0.056\n",
      "episode: 579/1001, score: 66, e: 0.055\n",
      "episode: 580/1001, score: 75, e: 0.055\n",
      "episode: 581/1001, score: 89, e: 0.055\n",
      "episode: 582/1001, score: 76, e: 0.055\n",
      "episode: 583/1001, score: 199, e: 0.054\n",
      "episode: 584/1001, score: 168, e: 0.054\n",
      "episode: 585/1001, score: 88, e: 0.054\n",
      "episode: 586/1001, score: 199, e: 0.054\n",
      "episode: 587/1001, score: 33, e: 0.053\n",
      "episode: 588/1001, score: 142, e: 0.053\n",
      "episode: 589/1001, score: 199, e: 0.053\n",
      "episode: 590/1001, score: 117, e: 0.052\n",
      "episode: 591/1001, score: 80, e: 0.052\n",
      "episode: 592/1001, score: 61, e: 0.052\n",
      "episode: 593/1001, score: 72, e: 0.052\n",
      "episode: 594/1001, score: 87, e: 0.051\n",
      "episode: 595/1001, score: 78, e: 0.051\n",
      "episode: 596/1001, score: 80, e: 0.051\n",
      "episode: 597/1001, score: 115, e: 0.051\n",
      "episode: 598/1001, score: 87, e: 0.05\n",
      "episode: 599/1001, score: 115, e: 0.05\n",
      "episode: 600/1001, score: 137, e: 0.05\n",
      "episode: 601/1001, score: 79, e: 0.05\n",
      "episode: 602/1001, score: 61, e: 0.049\n",
      "episode: 603/1001, score: 81, e: 0.049\n",
      "episode: 604/1001, score: 132, e: 0.049\n",
      "episode: 605/1001, score: 169, e: 0.049\n",
      "episode: 606/1001, score: 127, e: 0.048\n",
      "episode: 607/1001, score: 199, e: 0.048\n",
      "episode: 608/1001, score: 199, e: 0.048\n",
      "episode: 609/1001, score: 199, e: 0.048\n",
      "episode: 610/1001, score: 199, e: 0.047\n",
      "episode: 611/1001, score: 125, e: 0.047\n",
      "episode: 612/1001, score: 190, e: 0.047\n",
      "episode: 613/1001, score: 167, e: 0.047\n",
      "episode: 614/1001, score: 127, e: 0.047\n",
      "episode: 615/1001, score: 153, e: 0.046\n",
      "episode: 616/1001, score: 199, e: 0.046\n",
      "episode: 617/1001, score: 199, e: 0.046\n",
      "episode: 618/1001, score: 123, e: 0.046\n",
      "episode: 619/1001, score: 149, e: 0.045\n",
      "episode: 620/1001, score: 199, e: 0.045\n",
      "episode: 621/1001, score: 199, e: 0.045\n",
      "episode: 622/1001, score: 110, e: 0.045\n",
      "episode: 623/1001, score: 199, e: 0.044\n",
      "episode: 624/1001, score: 147, e: 0.044\n",
      "episode: 625/1001, score: 199, e: 0.044\n",
      "episode: 626/1001, score: 199, e: 0.044\n",
      "episode: 627/1001, score: 135, e: 0.044\n",
      "episode: 628/1001, score: 102, e: 0.043\n",
      "episode: 629/1001, score: 146, e: 0.043\n",
      "episode: 630/1001, score: 101, e: 0.043\n",
      "episode: 631/1001, score: 62, e: 0.043\n",
      "episode: 632/1001, score: 82, e: 0.043\n",
      "episode: 633/1001, score: 199, e: 0.042\n",
      "episode: 634/1001, score: 113, e: 0.042\n",
      "episode: 635/1001, score: 162, e: 0.042\n",
      "episode: 636/1001, score: 188, e: 0.042\n",
      "episode: 637/1001, score: 199, e: 0.041\n",
      "episode: 638/1001, score: 96, e: 0.041\n",
      "episode: 639/1001, score: 199, e: 0.041\n",
      "episode: 640/1001, score: 199, e: 0.041\n",
      "episode: 641/1001, score: 199, e: 0.041\n",
      "episode: 642/1001, score: 24, e: 0.04\n",
      "episode: 643/1001, score: 46, e: 0.04\n",
      "episode: 644/1001, score: 185, e: 0.04\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 645/1001, score: 199, e: 0.04\n",
      "episode: 646/1001, score: 199, e: 0.04\n",
      "episode: 647/1001, score: 75, e: 0.039\n",
      "episode: 648/1001, score: 105, e: 0.039\n",
      "episode: 649/1001, score: 159, e: 0.039\n",
      "episode: 650/1001, score: 199, e: 0.039\n",
      "episode: 651/1001, score: 199, e: 0.039\n",
      "episode: 652/1001, score: 56, e: 0.038\n",
      "episode: 653/1001, score: 182, e: 0.038\n",
      "episode: 654/1001, score: 199, e: 0.038\n",
      "episode: 655/1001, score: 87, e: 0.038\n",
      "episode: 656/1001, score: 169, e: 0.038\n",
      "episode: 657/1001, score: 106, e: 0.038\n",
      "episode: 658/1001, score: 199, e: 0.037\n",
      "episode: 659/1001, score: 79, e: 0.037\n",
      "episode: 660/1001, score: 27, e: 0.037\n",
      "episode: 661/1001, score: 52, e: 0.037\n",
      "episode: 662/1001, score: 198, e: 0.037\n",
      "episode: 663/1001, score: 50, e: 0.036\n",
      "episode: 664/1001, score: 199, e: 0.036\n",
      "episode: 665/1001, score: 136, e: 0.036\n",
      "episode: 666/1001, score: 199, e: 0.036\n",
      "episode: 667/1001, score: 199, e: 0.036\n",
      "episode: 668/1001, score: 177, e: 0.035\n",
      "episode: 669/1001, score: 91, e: 0.035\n",
      "episode: 670/1001, score: 100, e: 0.035\n",
      "episode: 671/1001, score: 117, e: 0.035\n",
      "episode: 672/1001, score: 199, e: 0.035\n",
      "episode: 673/1001, score: 156, e: 0.035\n",
      "episode: 674/1001, score: 130, e: 0.034\n",
      "episode: 675/1001, score: 137, e: 0.034\n",
      "episode: 676/1001, score: 92, e: 0.034\n",
      "episode: 677/1001, score: 188, e: 0.034\n",
      "episode: 678/1001, score: 88, e: 0.034\n",
      "episode: 679/1001, score: 117, e: 0.034\n",
      "episode: 680/1001, score: 199, e: 0.033\n",
      "episode: 681/1001, score: 199, e: 0.033\n",
      "episode: 682/1001, score: 199, e: 0.033\n",
      "episode: 683/1001, score: 199, e: 0.033\n",
      "episode: 684/1001, score: 199, e: 0.033\n",
      "episode: 685/1001, score: 24, e: 0.033\n",
      "episode: 686/1001, score: 36, e: 0.032\n",
      "episode: 687/1001, score: 199, e: 0.032\n",
      "episode: 688/1001, score: 199, e: 0.032\n",
      "episode: 689/1001, score: 199, e: 0.032\n",
      "episode: 690/1001, score: 125, e: 0.032\n",
      "episode: 691/1001, score: 185, e: 0.032\n",
      "episode: 692/1001, score: 199, e: 0.031\n",
      "episode: 693/1001, score: 199, e: 0.031\n",
      "episode: 694/1001, score: 87, e: 0.031\n",
      "episode: 695/1001, score: 199, e: 0.031\n",
      "episode: 696/1001, score: 199, e: 0.031\n",
      "episode: 697/1001, score: 117, e: 0.031\n",
      "episode: 698/1001, score: 199, e: 0.031\n",
      "episode: 699/1001, score: 199, e: 0.03\n",
      "episode: 700/1001, score: 199, e: 0.03\n",
      "episode: 701/1001, score: 86, e: 0.03\n",
      "episode: 702/1001, score: 189, e: 0.03\n",
      "episode: 703/1001, score: 199, e: 0.03\n",
      "episode: 704/1001, score: 199, e: 0.03\n",
      "episode: 705/1001, score: 199, e: 0.029\n",
      "episode: 706/1001, score: 199, e: 0.029\n",
      "episode: 707/1001, score: 65, e: 0.029\n",
      "episode: 708/1001, score: 85, e: 0.029\n",
      "episode: 709/1001, score: 186, e: 0.029\n",
      "episode: 710/1001, score: 157, e: 0.029\n",
      "episode: 711/1001, score: 199, e: 0.029\n",
      "episode: 712/1001, score: 199, e: 0.028\n",
      "episode: 713/1001, score: 199, e: 0.028\n",
      "episode: 714/1001, score: 138, e: 0.028\n",
      "episode: 715/1001, score: 134, e: 0.028\n",
      "episode: 716/1001, score: 147, e: 0.028\n",
      "episode: 717/1001, score: 199, e: 0.028\n",
      "episode: 718/1001, score: 132, e: 0.028\n",
      "episode: 719/1001, score: 199, e: 0.027\n",
      "episode: 720/1001, score: 199, e: 0.027\n",
      "episode: 721/1001, score: 199, e: 0.027\n",
      "episode: 722/1001, score: 85, e: 0.027\n",
      "episode: 723/1001, score: 182, e: 0.027\n",
      "episode: 724/1001, score: 162, e: 0.027\n",
      "episode: 725/1001, score: 66, e: 0.027\n",
      "episode: 726/1001, score: 189, e: 0.027\n",
      "episode: 727/1001, score: 199, e: 0.026\n",
      "episode: 728/1001, score: 159, e: 0.026\n",
      "episode: 729/1001, score: 199, e: 0.026\n",
      "episode: 730/1001, score: 199, e: 0.026\n",
      "episode: 731/1001, score: 199, e: 0.026\n",
      "episode: 732/1001, score: 199, e: 0.026\n",
      "episode: 733/1001, score: 199, e: 0.026\n",
      "episode: 734/1001, score: 140, e: 0.025\n",
      "episode: 735/1001, score: 113, e: 0.025\n",
      "episode: 736/1001, score: 199, e: 0.025\n",
      "episode: 737/1001, score: 157, e: 0.025\n",
      "episode: 738/1001, score: 72, e: 0.025\n",
      "episode: 739/1001, score: 199, e: 0.025\n",
      "episode: 740/1001, score: 199, e: 0.025\n",
      "episode: 741/1001, score: 91, e: 0.025\n",
      "episode: 742/1001, score: 199, e: 0.024\n",
      "episode: 743/1001, score: 199, e: 0.024\n",
      "episode: 744/1001, score: 176, e: 0.024\n",
      "episode: 745/1001, score: 75, e: 0.024\n",
      "episode: 746/1001, score: 51, e: 0.024\n",
      "episode: 747/1001, score: 199, e: 0.024\n",
      "episode: 748/1001, score: 199, e: 0.024\n",
      "episode: 749/1001, score: 199, e: 0.024\n",
      "episode: 750/1001, score: 176, e: 0.024\n",
      "episode: 751/1001, score: 199, e: 0.023\n",
      "episode: 752/1001, score: 199, e: 0.023\n",
      "episode: 753/1001, score: 199, e: 0.023\n",
      "episode: 754/1001, score: 77, e: 0.023\n",
      "episode: 755/1001, score: 199, e: 0.023\n",
      "episode: 756/1001, score: 199, e: 0.023\n",
      "episode: 757/1001, score: 199, e: 0.023\n",
      "episode: 758/1001, score: 199, e: 0.023\n",
      "episode: 759/1001, score: 199, e: 0.022\n",
      "episode: 760/1001, score: 199, e: 0.022\n",
      "episode: 761/1001, score: 188, e: 0.022\n",
      "episode: 762/1001, score: 199, e: 0.022\n",
      "episode: 763/1001, score: 199, e: 0.022\n",
      "episode: 764/1001, score: 199, e: 0.022\n",
      "episode: 765/1001, score: 199, e: 0.022\n",
      "episode: 766/1001, score: 199, e: 0.022\n",
      "episode: 767/1001, score: 199, e: 0.022\n",
      "episode: 768/1001, score: 199, e: 0.022\n",
      "episode: 769/1001, score: 199, e: 0.021\n",
      "episode: 770/1001, score: 61, e: 0.021\n",
      "episode: 771/1001, score: 57, e: 0.021\n",
      "episode: 772/1001, score: 199, e: 0.021\n",
      "episode: 773/1001, score: 199, e: 0.021\n",
      "episode: 774/1001, score: 199, e: 0.021\n",
      "episode: 775/1001, score: 199, e: 0.021\n",
      "episode: 776/1001, score: 199, e: 0.021\n",
      "episode: 777/1001, score: 199, e: 0.021\n",
      "episode: 778/1001, score: 199, e: 0.02\n",
      "episode: 779/1001, score: 199, e: 0.02\n",
      "episode: 780/1001, score: 114, e: 0.02\n",
      "episode: 781/1001, score: 177, e: 0.02\n",
      "episode: 782/1001, score: 88, e: 0.02\n",
      "episode: 783/1001, score: 199, e: 0.02\n",
      "episode: 784/1001, score: 199, e: 0.02\n",
      "episode: 785/1001, score: 87, e: 0.02\n",
      "episode: 786/1001, score: 32, e: 0.02\n",
      "episode: 787/1001, score: 63, e: 0.02\n",
      "episode: 788/1001, score: 83, e: 0.019\n",
      "episode: 789/1001, score: 91, e: 0.019\n",
      "episode: 790/1001, score: 125, e: 0.019\n",
      "episode: 791/1001, score: 38, e: 0.019\n",
      "episode: 792/1001, score: 21, e: 0.019\n",
      "episode: 793/1001, score: 29, e: 0.019\n",
      "episode: 794/1001, score: 43, e: 0.019\n",
      "episode: 795/1001, score: 54, e: 0.019\n",
      "episode: 796/1001, score: 60, e: 0.019\n",
      "episode: 797/1001, score: 199, e: 0.019\n",
      "episode: 798/1001, score: 129, e: 0.019\n",
      "episode: 799/1001, score: 199, e: 0.018\n",
      "episode: 800/1001, score: 183, e: 0.018\n",
      "episode: 801/1001, score: 199, e: 0.018\n",
      "episode: 802/1001, score: 130, e: 0.018\n",
      "episode: 803/1001, score: 199, e: 0.018\n",
      "episode: 804/1001, score: 199, e: 0.018\n",
      "episode: 805/1001, score: 199, e: 0.018\n",
      "episode: 806/1001, score: 199, e: 0.018\n",
      "episode: 807/1001, score: 199, e: 0.018\n",
      "episode: 808/1001, score: 199, e: 0.018\n",
      "episode: 809/1001, score: 199, e: 0.018\n",
      "episode: 810/1001, score: 199, e: 0.017\n",
      "episode: 811/1001, score: 177, e: 0.017\n",
      "episode: 812/1001, score: 199, e: 0.017\n",
      "episode: 813/1001, score: 118, e: 0.017\n",
      "episode: 814/1001, score: 132, e: 0.017\n",
      "episode: 815/1001, score: 199, e: 0.017\n",
      "episode: 816/1001, score: 113, e: 0.017\n",
      "episode: 817/1001, score: 199, e: 0.017\n",
      "episode: 818/1001, score: 199, e: 0.017\n",
      "episode: 819/1001, score: 199, e: 0.017\n",
      "episode: 820/1001, score: 199, e: 0.017\n",
      "episode: 821/1001, score: 199, e: 0.016\n",
      "episode: 822/1001, score: 199, e: 0.016\n",
      "episode: 823/1001, score: 193, e: 0.016\n",
      "episode: 824/1001, score: 199, e: 0.016\n",
      "episode: 825/1001, score: 186, e: 0.016\n",
      "episode: 826/1001, score: 199, e: 0.016\n",
      "episode: 827/1001, score: 199, e: 0.016\n",
      "episode: 828/1001, score: 197, e: 0.016\n",
      "episode: 829/1001, score: 129, e: 0.016\n",
      "episode: 830/1001, score: 199, e: 0.016\n",
      "episode: 831/1001, score: 199, e: 0.016\n",
      "episode: 832/1001, score: 109, e: 0.016\n",
      "episode: 833/1001, score: 146, e: 0.016\n",
      "episode: 834/1001, score: 199, e: 0.015\n",
      "episode: 835/1001, score: 199, e: 0.015\n",
      "episode: 836/1001, score: 199, e: 0.015\n",
      "episode: 837/1001, score: 182, e: 0.015\n",
      "episode: 838/1001, score: 199, e: 0.015\n",
      "episode: 839/1001, score: 199, e: 0.015\n",
      "episode: 840/1001, score: 199, e: 0.015\n",
      "episode: 841/1001, score: 199, e: 0.015\n",
      "episode: 842/1001, score: 199, e: 0.015\n",
      "episode: 843/1001, score: 199, e: 0.015\n",
      "episode: 844/1001, score: 199, e: 0.015\n",
      "episode: 845/1001, score: 127, e: 0.015\n",
      "episode: 846/1001, score: 199, e: 0.015\n",
      "episode: 847/1001, score: 131, e: 0.014\n",
      "episode: 848/1001, score: 199, e: 0.014\n",
      "episode: 849/1001, score: 199, e: 0.014\n",
      "episode: 850/1001, score: 179, e: 0.014\n",
      "episode: 851/1001, score: 199, e: 0.014\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 852/1001, score: 199, e: 0.014\n",
      "episode: 853/1001, score: 199, e: 0.014\n",
      "episode: 854/1001, score: 199, e: 0.014\n",
      "episode: 855/1001, score: 173, e: 0.014\n",
      "episode: 856/1001, score: 199, e: 0.014\n",
      "episode: 857/1001, score: 96, e: 0.014\n",
      "episode: 858/1001, score: 199, e: 0.014\n",
      "episode: 859/1001, score: 199, e: 0.014\n",
      "episode: 860/1001, score: 199, e: 0.014\n",
      "episode: 861/1001, score: 196, e: 0.013\n",
      "episode: 862/1001, score: 199, e: 0.013\n",
      "episode: 863/1001, score: 168, e: 0.013\n",
      "episode: 864/1001, score: 199, e: 0.013\n",
      "episode: 865/1001, score: 199, e: 0.013\n",
      "episode: 866/1001, score: 136, e: 0.013\n",
      "episode: 867/1001, score: 199, e: 0.013\n",
      "episode: 868/1001, score: 199, e: 0.013\n",
      "episode: 869/1001, score: 199, e: 0.013\n",
      "episode: 870/1001, score: 199, e: 0.013\n",
      "episode: 871/1001, score: 199, e: 0.013\n",
      "episode: 872/1001, score: 198, e: 0.013\n",
      "episode: 873/1001, score: 199, e: 0.013\n",
      "episode: 874/1001, score: 199, e: 0.013\n",
      "episode: 875/1001, score: 199, e: 0.013\n",
      "episode: 876/1001, score: 199, e: 0.013\n",
      "episode: 877/1001, score: 199, e: 0.012\n",
      "episode: 878/1001, score: 93, e: 0.012\n",
      "episode: 879/1001, score: 199, e: 0.012\n",
      "episode: 880/1001, score: 199, e: 0.012\n",
      "episode: 881/1001, score: 199, e: 0.012\n",
      "episode: 882/1001, score: 199, e: 0.012\n",
      "episode: 883/1001, score: 199, e: 0.012\n",
      "episode: 884/1001, score: 199, e: 0.012\n",
      "episode: 885/1001, score: 197, e: 0.012\n",
      "episode: 886/1001, score: 175, e: 0.012\n",
      "episode: 887/1001, score: 199, e: 0.012\n",
      "episode: 888/1001, score: 153, e: 0.012\n",
      "episode: 889/1001, score: 199, e: 0.012\n",
      "episode: 890/1001, score: 199, e: 0.012\n",
      "episode: 891/1001, score: 199, e: 0.012\n",
      "episode: 892/1001, score: 199, e: 0.012\n",
      "episode: 893/1001, score: 199, e: 0.011\n",
      "episode: 894/1001, score: 199, e: 0.011\n",
      "episode: 895/1001, score: 168, e: 0.011\n",
      "episode: 896/1001, score: 192, e: 0.011\n",
      "episode: 897/1001, score: 199, e: 0.011\n",
      "episode: 898/1001, score: 199, e: 0.011\n",
      "episode: 899/1001, score: 126, e: 0.011\n",
      "episode: 900/1001, score: 199, e: 0.011\n",
      "episode: 901/1001, score: 143, e: 0.011\n",
      "episode: 902/1001, score: 199, e: 0.011\n",
      "episode: 903/1001, score: 199, e: 0.011\n",
      "episode: 904/1001, score: 199, e: 0.011\n",
      "episode: 905/1001, score: 199, e: 0.011\n",
      "episode: 906/1001, score: 199, e: 0.011\n",
      "episode: 907/1001, score: 134, e: 0.011\n",
      "episode: 908/1001, score: 194, e: 0.011\n",
      "episode: 909/1001, score: 199, e: 0.011\n",
      "episode: 910/1001, score: 199, e: 0.011\n",
      "episode: 911/1001, score: 150, e: 0.01\n",
      "episode: 912/1001, score: 147, e: 0.01\n",
      "episode: 913/1001, score: 172, e: 0.01\n",
      "episode: 914/1001, score: 151, e: 0.01\n",
      "episode: 915/1001, score: 199, e: 0.01\n",
      "episode: 916/1001, score: 199, e: 0.01\n",
      "episode: 917/1001, score: 101, e: 0.01\n",
      "episode: 918/1001, score: 199, e: 0.01\n",
      "episode: 919/1001, score: 199, e: 0.01\n",
      "episode: 920/1001, score: 152, e: 0.01\n",
      "episode: 921/1001, score: 88, e: 0.01\n",
      "episode: 922/1001, score: 72, e: 0.01\n",
      "episode: 923/1001, score: 126, e: 0.01\n",
      "episode: 924/1001, score: 199, e: 0.01\n",
      "episode: 925/1001, score: 144, e: 0.01\n",
      "episode: 926/1001, score: 163, e: 0.01\n",
      "episode: 927/1001, score: 150, e: 0.01\n",
      "episode: 928/1001, score: 141, e: 0.01\n",
      "episode: 929/1001, score: 199, e: 0.01\n",
      "episode: 930/1001, score: 170, e: 0.01\n",
      "episode: 931/1001, score: 120, e: 0.01\n",
      "episode: 932/1001, score: 126, e: 0.01\n",
      "episode: 933/1001, score: 148, e: 0.01\n",
      "episode: 934/1001, score: 131, e: 0.01\n",
      "episode: 935/1001, score: 137, e: 0.01\n",
      "episode: 936/1001, score: 187, e: 0.01\n",
      "episode: 937/1001, score: 199, e: 0.01\n",
      "episode: 938/1001, score: 159, e: 0.01\n",
      "episode: 939/1001, score: 127, e: 0.01\n",
      "episode: 940/1001, score: 165, e: 0.01\n",
      "episode: 941/1001, score: 137, e: 0.01\n",
      "episode: 942/1001, score: 199, e: 0.01\n",
      "episode: 943/1001, score: 199, e: 0.01\n",
      "episode: 944/1001, score: 199, e: 0.01\n",
      "episode: 945/1001, score: 82, e: 0.01\n",
      "episode: 946/1001, score: 121, e: 0.01\n",
      "episode: 947/1001, score: 199, e: 0.01\n",
      "episode: 948/1001, score: 110, e: 0.01\n",
      "episode: 949/1001, score: 128, e: 0.01\n",
      "episode: 950/1001, score: 147, e: 0.01\n",
      "episode: 951/1001, score: 199, e: 0.01\n",
      "episode: 952/1001, score: 155, e: 0.01\n",
      "episode: 953/1001, score: 177, e: 0.01\n",
      "episode: 954/1001, score: 151, e: 0.01\n",
      "episode: 955/1001, score: 135, e: 0.01\n",
      "episode: 956/1001, score: 199, e: 0.01\n",
      "episode: 957/1001, score: 163, e: 0.01\n",
      "episode: 958/1001, score: 199, e: 0.01\n",
      "episode: 959/1001, score: 199, e: 0.01\n",
      "episode: 960/1001, score: 124, e: 0.01\n",
      "episode: 961/1001, score: 146, e: 0.01\n",
      "episode: 962/1001, score: 199, e: 0.01\n",
      "episode: 963/1001, score: 199, e: 0.01\n",
      "episode: 964/1001, score: 148, e: 0.01\n",
      "episode: 965/1001, score: 133, e: 0.01\n",
      "episode: 966/1001, score: 136, e: 0.01\n",
      "episode: 967/1001, score: 199, e: 0.01\n",
      "episode: 968/1001, score: 178, e: 0.01\n",
      "episode: 969/1001, score: 198, e: 0.01\n",
      "episode: 970/1001, score: 164, e: 0.01\n",
      "episode: 971/1001, score: 172, e: 0.01\n",
      "episode: 972/1001, score: 184, e: 0.01\n",
      "episode: 973/1001, score: 148, e: 0.01\n",
      "episode: 974/1001, score: 84, e: 0.01\n",
      "episode: 975/1001, score: 199, e: 0.01\n",
      "episode: 976/1001, score: 107, e: 0.01\n",
      "episode: 977/1001, score: 112, e: 0.01\n",
      "episode: 978/1001, score: 117, e: 0.01\n",
      "episode: 979/1001, score: 134, e: 0.01\n",
      "episode: 980/1001, score: 114, e: 0.01\n",
      "episode: 981/1001, score: 126, e: 0.01\n",
      "episode: 982/1001, score: 80, e: 0.01\n",
      "episode: 983/1001, score: 199, e: 0.01\n",
      "episode: 984/1001, score: 167, e: 0.01\n",
      "episode: 985/1001, score: 106, e: 0.01\n",
      "episode: 986/1001, score: 140, e: 0.01\n",
      "episode: 987/1001, score: 149, e: 0.01\n",
      "episode: 988/1001, score: 199, e: 0.01\n",
      "episode: 989/1001, score: 199, e: 0.01\n",
      "episode: 990/1001, score: 199, e: 0.01\n",
      "episode: 991/1001, score: 199, e: 0.01\n",
      "episode: 992/1001, score: 57, e: 0.01\n",
      "episode: 993/1001, score: 47, e: 0.01\n",
      "episode: 994/1001, score: 109, e: 0.01\n",
      "episode: 995/1001, score: 199, e: 0.01\n",
      "episode: 996/1001, score: 199, e: 0.01\n",
      "episode: 997/1001, score: 171, e: 0.01\n",
      "episode: 998/1001, score: 199, e: 0.01\n",
      "episode: 999/1001, score: 199, e: 0.01\n",
      "episode: 1000/1001, score: 199, e: 0.01\n"
     ]
    }
   ],
   "source": [
    "done = False\n",
    "for e in range(n_episodes): # iterate over new episodes of the game\n",
    "    state = env.reset() # reset state at start of each new episode of the game\n",
    "    state = np.reshape(state, [1, state_size])\n",
    "    \n",
    "    for time in range(5000):  # time represents a frame of the game; goal is to keep pole upright as long as possible up to range, e.g., 500 or 5000 timesteps\n",
    "#         env.render()\n",
    "        action = agent.act(state) # action is either 0 or 1 (move cart left or right); decide on one or other here\n",
    "        next_state, reward, done, _ = env.step(action) # agent interacts with env, gets feedback; 4 state data points, e.g., pole angle, cart position        \n",
    "        reward = reward if not done else -10 # reward +1 for each additional frame with pole upright        \n",
    "        next_state = np.reshape(next_state, [1, state_size])\n",
    "        agent.remember(state, action, reward, next_state, done) # remember the previous timestep's state, actions, reward, etc.        \n",
    "        state = next_state # set \"current state\" for upcoming iteration to the current next state        \n",
    "        if done: # episode ends if agent drops pole or we reach timestep 5000\n",
    "            print(\"episode: {}/{}, score: {}, e: {:.2}\" # print the episode's score and agent's epsilon\n",
    "                  .format(e, n_episodes, time, agent.epsilon))\n",
    "            break # exit loop\n",
    "    if len(agent.memory) > batch_size:\n",
    "        agent.replay(batch_size) # train the agent by replaying the experiences of the episode\n",
    "    if e % 50 == 0:\n",
    "        agent.save(output_dir + \"weights_\" + '{:04d}'.format(e) + \".hdf5\")         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# saved agents can be loaded with agent.load(\"./path/filename.hdf5\") "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
